<!DOCTYPE html>
<html lang="zh-CN">
  <head>
    
    <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="generator" content="Hugo 0.68.3 with theme Tranquilpeak 0.4.7-BETA">
<meta name="author" content="唐宋缘明卿">
<meta name="keywords" content="git, 版本管理, development, next-gen">
<meta name="description" content="爬虫学的好，牢饭吃得饱。">


<meta property="og:description" content="爬虫学的好，牢饭吃得饱。">
<meta property="og:type" content="article">
<meta property="og:title" content="初识Scrapy框架">
<meta name="twitter:title" content="初识Scrapy框架">
<meta property="og:url" content="https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6/">
<meta property="twitter:url" content="https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6/">
<meta property="og:site_name" content="唐宋缘明卿的博客">
<meta property="og:description" content="爬虫学的好，牢饭吃得饱。">
<meta name="twitter:description" content="爬虫学的好，牢饭吃得饱。">
<meta property="og:locale" content="zh-cn">

  
    <meta property="article:published_time" content="2017-02-03T14:36:41">
  
  
    <meta property="article:modified_time" content="2017-02-03T14:36:41">
  
  
  
    
      <meta property="article:section" content="Django">
    
  
  
    
      <meta property="article:tag" content="合作开发">
    
      <meta property="article:tag" content="版本管理">
    
      <meta property="article:tag" content="git">
    
      <meta property="article:tag" content="仓库">
    
  


<meta name="twitter:card" content="summary">







  <meta property="og:image" content="//d1u9biwaxjngwg.cloudfront.net/welcome-to-tranquilpeak/city-750.jpg">
  <meta property="twitter:image" content="//d1u9biwaxjngwg.cloudfront.net/welcome-to-tranquilpeak/city-750.jpg">


  <meta property="og:image" content="//d1u9biwaxjngwg.cloudfront.net/welcome-to-tranquilpeak/city.jpg">
  <meta property="twitter:image" content="//d1u9biwaxjngwg.cloudfront.net/welcome-to-tranquilpeak/city.jpg">




  <meta property="og:image" content="http://img.qq22.com.cn/uploads/4134/gubtyqxfhz2.jpg?68513786">
  <meta property="twitter:image" content="http://img.qq22.com.cn/uploads/4134/gubtyqxfhz2.jpg?68513786">


    <title>初识Scrapy框架</title>

    <link rel="icon" href="https://gaohaihang.gitee.io/ghhs_technology_blog/favicon.png">
    

    

    <link rel="canonical" href="https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6/">

    
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.4/jquery.fancybox.min.css" integrity="sha256-vuXZ9LGmmwtjqFX1F+EKin1ThZMub58gKULUyf0qECk=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.4/helpers/jquery.fancybox-thumbs.min.css" integrity="sha256-SEa4XYAHihTcEP1f5gARTB2K26Uk8PsndQYHQC1f4jU=" crossorigin="anonymous" />
    
    
    <link rel="stylesheet" href="https://gaohaihang.gitee.io/ghhs_technology_blog/css/style-twzjdbqhmnnacqs0pwwdzcdbt8yhv8giawvjqjmyfoqnvazl0dalmnhdkvp7.min.css" />
    
    

    
      
    
    
  </head>

  <body>
    <div id="blog">
      <header id="header" data-behavior="5">
  <i id="btn-open-sidebar" class="fa fa-lg fa-bars"></i>
  <div class="header-title">
    <a class="header-title-link" href="https://gaohaihang.gitee.io/ghhs_technology_blog/">唐宋缘明卿的博客</a>
  </div>
  
    
      <a class="header-right-picture "
         href="https://gaohaihang.gitee.io/ghhs_technology_blog/#about">
    
    
    
      
        <img class="header-picture" src="http://img.qq22.com.cn/uploads/4134/gubtyqxfhz2.jpg?68513786" alt="作者的图片" />
      
    
    </a>
  
</header>

      <nav id="sidebar" data-behavior="5">
  <div class="sidebar-container">
    
      <div class="sidebar-profile">
        <a href="https://gaohaihang.gitee.io/ghhs_technology_blog/#about">
          <img class="sidebar-profile-picture" src="http://img.qq22.com.cn/uploads/4134/gubtyqxfhz2.jpg?68513786" alt="作者的图片" />
        </a>
        <h4 class="sidebar-profile-name">唐宋缘明卿</h4>
        
          <h5 class="sidebar-profile-bio">弱小的人，才习惯嘲讽与否定；内心强大的人，从不吝啬赞美与鼓励。</h5>
        
      </div>
    
    <ul class="sidebar-buttons">
      
  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://gaohaihang.gitee.io/ghhs_technology_blog/">
    
      <i class="sidebar-button-icon fa fa-lg fa-home"></i>
      
      <span class="sidebar-button-desc">首页</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://gaohaihang.gitee.io/ghhs_technology_blog/categories">
    
      <i class="sidebar-button-icon fa fa-lg fa-bookmark"></i>
      
      <span class="sidebar-button-desc">分类</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://gaohaihang.gitee.io/ghhs_technology_blog/tags">
    
      <i class="sidebar-button-icon fa fa-lg fa-tags"></i>
      
      <span class="sidebar-button-desc">标签</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://gaohaihang.gitee.io/ghhs_technology_blog/archives">
    
      <i class="sidebar-button-icon fa fa-lg fa-archive"></i>
      
      <span class="sidebar-button-desc">归档</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://gaohaihang.gitee.io/ghhs_technology_blog/resume">
    
      <i class="sidebar-button-icon fa fa-address-card"></i>
      
      <span class="sidebar-button-desc">Resume</span>
    </a>
  </li>


    </ul>
    <ul class="sidebar-buttons">
      
  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://github.com/ghh617415200" target="_blank" rel="noopener">
    
      <i class="sidebar-button-icon fa fa-lg fa-github"></i>
      
      <span class="sidebar-button-desc">GitHub</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://stackoverflow.com/users/13559927/ghh-cirs" target="_blank" rel="noopener">
    
      <i class="sidebar-button-icon fa fa-lg fa-stack-overflow"></i>
      
      <span class="sidebar-button-desc">Stack Overflow</span>
    </a>
  </li>


    </ul>
    <ul class="sidebar-buttons">
      
  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://gaohaihang.gitee.io/ghhs_technology_blog/index.xml">
    
      <i class="sidebar-button-icon fa fa-lg fa-rss"></i>
      
      <span class="sidebar-button-desc">Rss</span>
    </a>
  </li>


    </ul>
  </div>
</nav>

      
  <div class="post-header-cover
              text-center
              "
       style="background-image:url('//d1u9biwaxjngwg.cloudfront.net/welcome-to-tranquilpeak/city.jpg')"
       data-behavior="5">
    
      <div class="post-header main-content-wrap text-center">
  
    <h1 class="post-title" itemprop="headline">
      初识Scrapy框架
    </h1>
  
  
  <div class="postShorten-meta post-meta">
    
      <time itemprop="datePublished" datetime="2017-02-03T14:36:41&#43;08:00">
        
  二月 3, 2017

      </time>
    
    
  
  
    <span>发布在</span>
    
      <a class="category-link" href="https://gaohaihang.gitee.io/ghhs_technology_blog/categories/django">Django</a>
    
  

  </div>

</div>
    
  </div>


      <div id="main" data-behavior="5"
        class="hasCover
               hasCoverMetaIn
               ">
        <article class="post" itemscope itemType="http://schema.org/BlogPosting">
          
          
          <div class="post-content markdown" itemprop="articleBody">
            <div class="main-content-wrap">
              <p>爬虫学的好，牢饭吃得饱。</p>
<h2 id="一scrapy安装与环境依赖">一、scrapy安装与环境依赖</h2>
<h3 id="1在安装scrapy前需要安装好相应的依赖库-再安装scrapy-具体安装步骤如下">1.在安装scrapy前需要安装好相应的依赖库, 再安装scrapy, 具体安装步骤如下:</h3>
<pre><code>(1).安装lxml库: pip install lxml
(2).安装wheel: pip install wheel
(3).安装twisted: pip install twisted文件路径
    (twisted需下载后本地安装,下载地址:http://www.lfd.uci.edu/~gohlke/pythonlibs/#twisted)
    (版本选择如下图,版本后面有解释,请根据自己实际选择)
(4).安装pywin32: pip install pywin32
    (注意:以上安装步骤一定要确保每一步安装都成功,没有报错信息,如有报错自行百度解决)
(5).安装scrapy: pip install scrapy
    (注意:以上安装步骤一定要确保每一步安装都成功,没有报错信息,如有报错自行百度解决)
(6).成功验证:在cmd命令行输入scrapy,显示Scrapy1.6.0-no active project,证明安装成功 
</code></pre>
<p><img src="https://gitee.com/gaohaihang/ghhs_technology_blog/raw/master/img/res/scrapy1.png" alt="img"></p>
<hr>
<h3 id="2创建项目">2.创建项目</h3>
<pre><code>1.手动创建一个目录test
2.在test文件夹下创建爬虫项目为spiderpro: scrapy startproject spiderpro
3.进入项目文件夹: cd spiderpro
4.创建爬虫文件: scrapy genspider 爬虫名 域名
</code></pre><hr>
<h3 id="3项目目录介绍">3.项目目录介绍</h3>
<pre><code>-spiderpro
----spiderpro # 项目目录
------__init__
------spiders:          爬虫文件目录
--------__init__
--------tests.py:       爬虫文件
------items.py:         定义爬取数据持久化的数据结构
------middlewares.py:   定义中间件
------pipelines.py:     管道,持久化存储相关
------settings.py:      配置文件
　　venv:               虚拟环境目录
　 scrapy.cfg:          scrapy项目配置文件
</code></pre><p><strong>说明:</strong></p>
<ol>
<li>spiders:其内包含一个个Spider的实现, 每个Spider是一个单独的文件</li>
<li>items.py:它定义了Item数据结构, 爬取到的数据存储为哪些字段</li>
<li>pipelines.py:它定义Item Pipeline的实现</li>
<li>settings.py:项目的全局配置</li>
<li>middlewares.py:定义中间件, 包括爬虫中间件和下载中间件</li>
<li>scrapy.cfg:它是scrapy项目的配置文件, 其内定义了项目的配置  路径, 部署相关的信息等</li>
</ol>
<hr>
<h3 id="4scrapy框架介绍-5大核心组件与数据流向">4.scrapy框架介绍: 5大核心组件与数据流向</h3>
<p><img src="https://gitee.com/gaohaihang/ghhs_technology_blog/raw/master/img/res/scrapy2.png" alt="img"></p>
<h4 id="1架构">(1)架构:</h4>
<ul>
<li>
<p>Scrapy Engine: 这是引擎，负责Spiders、ItemPipeline、Downloader、Scheduler中间的通讯，信号、数据传递等等!</p>
</li>
<li>
<p>Scheduler(调度器): 它负责接受引擎发送过来的requests请求，并按照一定的方式进行整理排列，入队、并等待Scrapy Engine(引擎)来请求时，交给引擎。</p>
</li>
<li>
<p>Downloader（下载器)：负责下载Scrapy Engine(引擎)发送的所有Requests请求，并将其获取到的Responses交还给Scrapy Engine(引擎)，由引擎交给Spiders来处理</p>
</li>
<li>
<p>Spiders：它负责处理所有Responses,从中分析提取数据，获取Item字段需要的数据，并将需要跟进的URL提交给引擎，再次进入Scheduler(调度器)</p>
</li>
<li>
<p>Item Pipeline：它负责处理Spiders中获取到的Item，并进行处理，比如去重，持久化存储（存数据库，写入文件，总之就是保存数据用的）</p>
</li>
<li>
<p>Downloader Middlewares(下载中间件)：你可以当作是一个可以自定义扩展下载功能的组件</p>
</li>
<li>
<p>Spider Middlewares(Spider中间件)：你可以理解为是一个可以自定扩展和操作引擎和Spiders中间‘通信‘的功能组件（比如进入Spiders的Responses;和从Spiders出去的Requests）</p>
</li>
</ul>
<h4 id="2工作流">(2).工作流:</h4>
<ol>
<li>
<p>spider将请求发送给引擎, 引擎将request发送给调度器进行请求调度</p>
</li>
<li>
<p>调度器把接下来要请求的request发送给引擎, 引擎传递给下载器, 中间会途径下载中间件</p>
</li>
<li>
<p>下载携带request访问服务器, 并将爬取内容response返回给引擎, 引擎将response返回给spider</p>
</li>
<li>
<p>spider将response传递给自己的parse进行数据解析处理及构建item一系列的工作, 最后将item返回给引擎, 引擎传递个pipeline</p>
</li>
<li>
<p>pipe获取到item后进行数据持久化</p>
</li>
<li>
<p>以上过程不断循环直至爬虫程序终止</p>
</li>
</ol>
<hr>
<p>下面以爬取开源中国博客作为简单的scrapy爬虫实例</p>
<pre><code># 创建项目:
scrapy startproject kycn # 创建项目
cd kycn # 切换到项目目录
scrapy genspider ky 
</code></pre><pre><code># item文件定义数据存储的字段:
import scrapy
class KycnItem(scrapy.Item):
    # define the fields for your item here like:
    title = scrapy.Field()
    brief = scrapy.Field()
    author = scrapy.Field()
    date = scrapy.Field()
    see = scrapy.Field()
    comment = scrapy.Field()
    like = scrapy.Field()
    typename = scrapy.Field()
    imgname = scrapy.Field()
</code></pre><pre><code># spider文件中定义解析数据的方法
from ..items import KycnItem

class KySpider(scrapy.Spider):
    name = 'ky'
    # allowed_domains = ['baidu.com']
    start_urls = ['https://www.oschina.net/blog']

    def imgdown(self,response):
        item=response.meta['item']
        # with open(&quot;C:/Users/Desktop/images/kyitem2/{}&quot;.format(item[&quot;imgname&quot;]),&quot;wb&quot;) as f :
        #     f.write(response.body)
        yield item

    def detailparse(self,response):
        typ = response.meta[&quot;typ&quot;]
        div_list = response.xpath('//div[@class=&quot;item blog-item&quot;]')
        # print(div_list)
        for div in div_list:
            title = div.xpath(&quot;.//a[@class='header']/@title&quot;).extract_first()
            brief = div.xpath(&quot;.//div[@class='description']/p/text()&quot;).extract_first()
            author = div.xpath(&quot;.//div[@class='ui horizontal list']/div[1]/a/text()&quot;).extract_first()
            date = div.xpath(&quot;.//div[@class='ui horizontal list']/div[2]/text()&quot;).extract_first()
            see = div.xpath(&quot;.//div[@class='ui horizontal list']/div[3]/text()&quot;).extract_first()
            comment = div.xpath(&quot;.//div[@class='ui horizontal list']/div[4]/a/text()&quot;).extract_first()
            like = div.xpath(&quot;.//div[@class='ui horizontal list']/div[5]/text()&quot;).extract_first()
            typename = typ
            

            img_url = div.xpath(&quot;.//a[@class='ui small image']/img/@src&quot;).extract_first()
            if all([title,brief,author,date,see,comment,like,typename,img_url]):
                item = KycnItem()
                item[&quot;title&quot;]=title
                item[&quot;brief&quot;]=brief
                item[&quot;author&quot;]=author
                item[&quot;date&quot;]=date
                item[&quot;see&quot;]=see
                item[&quot;comment&quot;]=comment
                item[&quot;like&quot;]=like
                item[&quot;typename&quot;]=typename
                item[&quot;imgname&quot;] = img_url.split(&quot;/&quot;)[-1]
                yield scrapy.Request(url=img_url,callback=self.imgdown,meta={&quot;item&quot;:item})


    def parse(self, response):
        div = response.xpath('//div[@class=&quot;two wide column computer only left-channel&quot;]/div[1]/a/@href').extract()[1:]
        title = response.xpath('//div[@class=&quot;two wide column computer only left-channel&quot;]/div[1]/a/text()').extract()[1:]
        # print(div,title)
        for i in div:
            yield scrapy.Request(url=i,callback=self.detailparse,meta={&quot;typ&quot;:title[div.index(i)]})
</code></pre><pre><code># 在pipeline中定义管道类进行数据的存储
import pymongo

class KycnPipeline(object):
    conn = pymongo.MongoClient()
    db = conn.kycn
    table = db.table2

    def process_item(self, item, spider):
        self.table.insert_one(dict(item))
        return item
</code></pre><pre><code># 此示例中配置文件中的配置的项, 注意是不是全部的配置, 是针对该项目增加或修改的配置项

# 忽略robots协议
ROBOTSTXT_OBEY = False

# UA伪装
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'


# 管道类的注册配置
ITEM_PIPELINES ={
    'kycn.pipelines.KycnPipeline': 300,
}
</code></pre><hr>
              
            </div>
          </div>
          <div id="post-footer" class="post-footer main-content-wrap">
            
              
                
                
                  <div class="post-footer-tags">
                    <span class="text-color-light text-small">标签</span><br/>
                    
  <a class="tag tag--primary tag--small" href="https://gaohaihang.gitee.io/ghhs_technology_blog/tags/%E5%90%88%E4%BD%9C%E5%BC%80%E5%8F%91/">合作开发</a>

  <a class="tag tag--primary tag--small" href="https://gaohaihang.gitee.io/ghhs_technology_blog/tags/%E7%89%88%E6%9C%AC%E7%AE%A1%E7%90%86/">版本管理</a>

  <a class="tag tag--primary tag--small" href="https://gaohaihang.gitee.io/ghhs_technology_blog/tags/git/">git</a>

  <a class="tag tag--primary tag--small" href="https://gaohaihang.gitee.io/ghhs_technology_blog/tags/%E4%BB%93%E5%BA%93/">仓库</a>

                  </div>
                
              
            
            <div class="post-actions-wrap">
  
      <nav >
        <ul class="post-actions post-action-nav">
          
            <li class="post-action">
              
                <a class="post-action-btn btn btn--default tooltip--top" href="https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86celery%E5%BC%82%E6%AD%A5%E9%82%AE%E7%AE%B1%E9%AA%8C%E8%AF%81/" data-tooltip="初识Celery&amp;异步邮箱验证">
              
                  <i class="fa fa-angle-left"></i>
                  <span class="hide-xs hide-sm text-small icon-ml">下一篇</span>
                </a>
            </li>
            <li class="post-action">
              
                <a class="post-action-btn btn btn--default tooltip--top" href="https://gaohaihang.gitee.io/ghhs_technology_blog/2016/07/git%E4%B9%8B%E4%B8%80%E9%94%AE%E4%B8%89%E8%BF%9E%E5%90%88%E4%BD%9C%E5%BC%80%E5%8F%91/" data-tooltip="Git之一键三连&amp;合作开发">
              
                  <span class="hide-xs hide-sm text-small icon-mr">上一篇</span>
                  <i class="fa fa-angle-right"></i>
                </a>
            </li>
          
        </ul>
      </nav>
    <ul class="post-actions post-action-share" >
      
        <li class="post-action hide-lg hide-md hide-sm">
          <a class="post-action-btn btn btn--default btn-open-shareoptions" href="#btn-open-shareoptions">
            <i class="fa fa-share-alt"></i>
          </a>
        </li>
        
          <li class="post-action hide-xs">
            <a class="post-action-btn btn btn--default" target="new" href="https://www.facebook.com/sharer/sharer.php?u=https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6/">
              <i class="fa fa-facebook-official"></i>
            </a>
          </li>
        
          <li class="post-action hide-xs">
            <a class="post-action-btn btn btn--default" target="new" href="https://twitter.com/intent/tweet?text=https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6/">
              <i class="fa fa-twitter"></i>
            </a>
          </li>
        
          <li class="post-action hide-xs">
            <a class="post-action-btn btn btn--default" target="new" href="https://plus.google.com/share?url=https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6/">
              <i class="fa fa-google-plus"></i>
            </a>
          </li>
        
      
      
        <li class="post-action">
          <a class="post-action-btn btn btn--default" href="#disqus_thread">
            <i class="fa fa-comment-o"></i>
          </a>
        </li>
      
      <li class="post-action">
        
          <a class="post-action-btn btn btn--default" href="#">
        
          <i class="fa fa-list"></i>
        </a>
      </li>
    </ul>
  
</div>

            
              
                <div id="disqus_thread">
  <noscript>Please enable JavaScript to view the <a href="//disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
</div>
              
            
          </div>
        </article>
        <footer id="footer" class="main-content-wrap">
  <span class="copyrights">
    &copy; 2020 唐宋缘明卿. All Rights Reserved
  </span>
</footer>

      </div>
      <div id="bottom-bar" class="post-bottom-bar" data-behavior="5">
        <div class="post-actions-wrap">
  
      <nav >
        <ul class="post-actions post-action-nav">
          
            <li class="post-action">
              
                <a class="post-action-btn btn btn--default tooltip--top" href="https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86celery%E5%BC%82%E6%AD%A5%E9%82%AE%E7%AE%B1%E9%AA%8C%E8%AF%81/" data-tooltip="初识Celery&amp;异步邮箱验证">
              
                  <i class="fa fa-angle-left"></i>
                  <span class="hide-xs hide-sm text-small icon-ml">下一篇</span>
                </a>
            </li>
            <li class="post-action">
              
                <a class="post-action-btn btn btn--default tooltip--top" href="https://gaohaihang.gitee.io/ghhs_technology_blog/2016/07/git%E4%B9%8B%E4%B8%80%E9%94%AE%E4%B8%89%E8%BF%9E%E5%90%88%E4%BD%9C%E5%BC%80%E5%8F%91/" data-tooltip="Git之一键三连&amp;合作开发">
              
                  <span class="hide-xs hide-sm text-small icon-mr">上一篇</span>
                  <i class="fa fa-angle-right"></i>
                </a>
            </li>
          
        </ul>
      </nav>
    <ul class="post-actions post-action-share" >
      
        <li class="post-action hide-lg hide-md hide-sm">
          <a class="post-action-btn btn btn--default btn-open-shareoptions" href="#btn-open-shareoptions">
            <i class="fa fa-share-alt"></i>
          </a>
        </li>
        
          <li class="post-action hide-xs">
            <a class="post-action-btn btn btn--default" target="new" href="https://www.facebook.com/sharer/sharer.php?u=https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6/">
              <i class="fa fa-facebook-official"></i>
            </a>
          </li>
        
          <li class="post-action hide-xs">
            <a class="post-action-btn btn btn--default" target="new" href="https://twitter.com/intent/tweet?text=https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6/">
              <i class="fa fa-twitter"></i>
            </a>
          </li>
        
          <li class="post-action hide-xs">
            <a class="post-action-btn btn btn--default" target="new" href="https://plus.google.com/share?url=https://gaohaihang.gitee.io/ghhs_technology_blog/2017/02/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6/">
              <i class="fa fa-google-plus"></i>
            </a>
          </li>
        
      
      
        <li class="post-action">
          <a class="post-action-btn btn btn--default" href="#disqus_thread">
            <i class="fa fa-comment-o"></i>
          </a>
        </li>
      
      <li class="post-action">
        
          <a class="post-action-btn btn btn--default" href="#">
        
          <i class="fa fa-list"></i>
        </a>
      </li>
    </ul>
  
</div>

      </div>
      <div id="share-options-bar" class="share-options-bar" data-behavior="5">
  <i id="btn-close-shareoptions" class="fa fa-close"></i>
  <ul class="share-options">
    
      <li class="share-option">
        <a class="share-option-btn" target="new" href="https://www.facebook.com/sharer/sharer.php?u=https%3A%2F%2Fgaohaihang.gitee.io%2Fghhs_technology_blog%2F2017%2F02%2F%25E5%2588%259D%25E8%25AF%2586scrapy%25E6%25A1%2586%25E6%259E%25B6%2F">
          <i class="fa fa-facebook-official"></i><span>分享到 Facebook</span>
        </a>
      </li>
    
      <li class="share-option">
        <a class="share-option-btn" target="new" href="https://twitter.com/intent/tweet?text=https%3A%2F%2Fgaohaihang.gitee.io%2Fghhs_technology_blog%2F2017%2F02%2F%25E5%2588%259D%25E8%25AF%2586scrapy%25E6%25A1%2586%25E6%259E%25B6%2F">
          <i class="fa fa-twitter"></i><span>分享到 Twitter</span>
        </a>
      </li>
    
      <li class="share-option">
        <a class="share-option-btn" target="new" href="https://plus.google.com/share?url=https%3A%2F%2Fgaohaihang.gitee.io%2Fghhs_technology_blog%2F2017%2F02%2F%25E5%2588%259D%25E8%25AF%2586scrapy%25E6%25A1%2586%25E6%259E%25B6%2F">
          <i class="fa fa-google-plus"></i><span>分享到 Google&#43;</span>
        </a>
      </li>
    
  </ul>
</div>
<div id="share-options-mask" class="share-options-mask"></div>
    </div>
    
    <div id="about">
  <div id="about-card">
    <div id="about-btn-close">
      <i class="fa fa-remove"></i>
    </div>
    
      <img id="about-card-picture" src="http://img.qq22.com.cn/uploads/4134/gubtyqxfhz2.jpg?68513786" alt="作者的图片" />
    
    <h4 id="about-card-name">唐宋缘明卿</h4>
    
      <div id="about-card-bio">弱小的人，才习惯嘲讽与否定；内心强大的人，从不吝啬赞美与鼓励。</div>
    
    
      <div id="about-card-job">
        <i class="fa fa-briefcase"></i>
        <br/>
        python全栈工程师
      </div>
    
    
      <div id="about-card-location">
        <i class="fa fa-map-marker"></i>
        <br/>
        中国 北京
      </div>
    
  </div>
</div>

    

    
  
    
      <div id="cover" style="background-image:url('https://gaohaihang.gitee.io/ghhs_technology_blog/images/cover.jpg');"></div>
    
  


    
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.4/jquery.min.js" integrity="sha256-BbhdlvQf/xTY9gja0Dq3HiwQF8LaCRTXxZKRutelT44=" crossorigin="anonymous"></script>

  <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js" integrity="sha256-/BfiIkHlHoVihZdc6TFuj7MmJ0TWcWsMXkeDFwhi0zw=" crossorigin="anonymous"></script>

<script src="https://cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.7/js/jquery.fancybox.min.js" integrity="sha256-GEAnjcTqVP+vBp3SSc8bEDQqvWAZMiHyUSIorrWwH50=" crossorigin="anonymous"></script>


<script src="https://gaohaihang.gitee.io/ghhs_technology_blog/js/script-pcw6v3xilnxydl1vddzazdverrnn9ctynvnxgwho987mfyqkuylcb1nlt.min.js"></script>


<script lang="javascript">
window.onload = updateMinWidth;
window.onresize = updateMinWidth;
document.getElementById("sidebar").addEventListener("transitionend", updateMinWidth);
function updateMinWidth() {
  var sidebar = document.getElementById("sidebar");
  var main = document.getElementById("main");
  main.style.minWidth = "";
  var w1 = getComputedStyle(main).getPropertyValue("min-width");
  var w2 = getComputedStyle(sidebar).getPropertyValue("width");
  var w3 = getComputedStyle(sidebar).getPropertyValue("left");
  main.style.minWidth = `calc(${w1} - ${w2} - ${w3})`;
}
</script>

<script>
$(document).ready(function() {
  hljs.configure({ classPrefix: '', useBR: false });
  $('pre.code-highlight > code, pre > code').each(function(i, block) {
    if (!$(this).hasClass('codeblock')) {
      $(this).addClass('codeblock');
    }
    hljs.highlightBlock(block);
  });
});
</script>


  
    
      <script>
        var disqus_config = function () {
          this.page.url = 'https:\/\/gaohaihang.gitee.io\/ghhs_technology_blog\/2017\/02\/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6\/';
          
            this.page.identifier = '\/2017\/02\/%E5%88%9D%E8%AF%86scrapy%E6%A1%86%E6%9E%B6\/'
          
        };
        (function() {
          
          
          if (window.location.hostname == "localhost") {
            return;
          }
          var d = document, s = d.createElement('script');
          var disqus_shortname = 'hugo-tranquilpeak-theme';
          s.src = '//' + disqus_shortname + '.disqus.com/embed.js';

          s.setAttribute('data-timestamp', +new Date());
          (d.head || d.body).appendChild(s);
        })();
      </script>
    
  




    
  </body>
</html>

